import os
import sys
import time
import re
import shutil
from PyQt4 import QtCore, QtGui, Qt
from Bio import Entrez, Medline
import pyPdf


class IndexWindow(QtGui.QDialog):
    '''
    A dialog widget used for the indexing window. Subclassed for its close event.
    '''
    def __init__(self, parent = None):
        QtGui.QDialog.__init__(self, parent)
        self.setAcceptDrops(True)
        
    def closeEvent(self, event):
        self.emit(QtCore.SIGNAL('windowClosed'))


        
class PDFIndexingThread(QtCore.QThread):
    '''
    Main thread that does pdf indexing.
    It uses the scopus website for doi based pdf identification.
    See individual methods for more details.
    '''
    def __init__(self, dir_name, config_dict, timeN, parent = None):
        '''
        On initializing, save the directory name, configuration dictionary and scopus id.
        Additionally, get a scopus url pre and post. The reason to do it like this instead of %s is
        because the id might contain certain characters that could pose a problem.
        Also start for each thread a running detection. If either the calling window (the IndexWindow above)
        is closed, destroyed or okayed.
        '''
        QtCore.QThread.__init__(self, parent)
        self.dir_name = dir_name
        self.config_dict = config_dict
        self.time_st = timeN
        self.counter = 0
        self.running = True

    def __del__(self):
        '''
        Allow graceful exit.
        '''
        self.wait()
    
    def start_process(self):
        '''
        A start process so that arguments can be taken in if needed.
        '''
        self.start()
        
    def run(self):
        '''
        Start the thread, basically calling get_pdf_paths, passing in the directory name.
        '''
        self.get_pdf_paths(self.dir_name)

    def get_pdf_paths(self, dir_name):
        '''
        Main function of this thread.
        1. Import required files
        2. Create some regular expresssions for the body and id values
        3. Recursively walk down the argument directory
        4. If a pdf file is encountered, get its doi number
        5. Construct a new url with this doi number
        6. This url should have the same listing as in a citations page. So do the same.
        Create a RefCiteRetrieval object.
        7. Get the record and add the required id.
        8. Convert the pdf again to text and save the record details.
        
        '''
        #1. 
        import pdf_downloader
        import reference_retrieve
        
        #2. 
        pdf_re = re.compile('pdf$', re.IGNORECASE)
        self.tempdir = self.config_dict['storageLocation'] + 'temp/'
        if not os.path.isdir(self.tempdir):
            os.mkdir(self.tempdir)
        #3.
        self.emit(QtCore.SIGNAL('pdf_file'), '<b><font size="3">Getting pdf files</font></b><br><br>')
        for root, subFolders, files in os.walk(dir_name):
            for filename in files:
                #4. 
                if len(pdf_re.findall(filename)) == 1:
                    if self.running:
                        self.counter += 1
                        pdf_filename = root + '/' + filename
                        #4. 
                        base_filename = filename[:-4]
                        self.emit(QtCore.SIGNAL('pdf_file'),
                                      '<font color="#330000" size="3"><b>File: </b>%s</font><br>' % pdf_filename)
                        try:
                            x = self.index_previously_indexed(root, base_filename, pdf_filename)
                            if x:
                                continue
                        except:
                            pass
                        doi_val = self.get_doi(pdf_filename)
                        if not doi_val:
                            self.emit(QtCore.SIGNAL('pdf_file'),
                                      '- No doi with pyPdf<br>- Lets try something else, maybe the title or a line from the abstract.<br>')
                            # reusing doi_val variable.
                            doi_val = self.try_to_get_title(pdf_filename)
                            self.emit(QtCore.SIGNAL('pdf_file'),
                                      '- "%s"<br>' % doi_val)
                            search_term = doi_val
                        else:
                            self.emit(QtCore.SIGNAL('pdf_file'), '- <i>%s</i><br>' % doi_val)
                            search_term = '%s [aid]' % doi_val
                            #5. 
                        self.emit(QtCore.SIGNAL('pdf_file'),
                                  '- <i>%s</i><br>' % 'Accessing pubmed to get details.')
                        #6. 
                        try:
                            self.emit(QtCore.SIGNAL('pdf_file'), '- <i>%s</i><br>' % 'Extracting details.')
                            outputList = self.get_one_search_item(search_term)
                            if len(outputList) == 0: raise
                            outputList = outputList[0]
                            #7. 
                            eid_val = outputList['PMRecord']
                            if self.config_dict['tags_value'] == 'tagFullPath':
                                outputList['Tags'] = pdf_filename
                            elif self.config_dict['tags_value'] == 'tagDir':
                                outputList['Tags'] = pdf_filename.split('/')[-2]
                            elif self.config_dict['tags_value'] == 'tagCustom':
                                if not self.config_dict['tags_text'] == 'None':
                                    outputList['Tags'] = self.config_dict['tags_text']
                            else:
                                outputList['Tags'] = ''

                            #8.
                            if os.path.isfile('%s%s.dxt' % (self.config_dict['storageLocation'], eid_val)):
                                if 'Yes' in self.config_dict['markDuplicates']:
                                    self.emit(QtCore.SIGNAL('dup_pdf_file'),
                                              '<i>%s</i><br>' % pdf_filename)
                                    self.emit(QtCore.SIGNAL('pdf_file'),
                                          '-- <b><i>%s</i></b><br><br>' % 'Duplicate not overwritten.')
                                    continue
                                else:
                                    self.emit(QtCore.SIGNAL('pdf_file'),
                                          '-- <b><i>%s</i></b><br><br>' % 'Overwriting duplicates.')
                            osFunc = "%s %s -" % (self.config_dict['pdfToTextLocation'], pdf_filename)
                            x = os.popen(osFunc)
                            x = x.read()
                            x = x.split('. ')
                            x = '\n'.join(x)
                            self.emit(QtCore.SIGNAL('pdf_file'),
                                      '-- <i>%s</i><br><br>' % 'Saving searchable text.')
                            if self.config_dict['savePdfsCheck'] == 'Yes':
                               self.emit(QtCore.SIGNAL('pdf_file'),
                                      '-- <i>%s</i><br><br>' % 'Saving pdf to storage.') 
                               new_pdf_name = '%s%s.pdf' % (
                                   self.config_dict['storageLocation'], eid_val)
                               shutil.copyfile(pdf_filename, new_pdf_name)
                            f = open('%s%s.txt' % (self.config_dict['storageLocation'], eid_val), 'w')
                            f.write(x); f.close()
                            f = open('%s%s.dxt' % (self.config_dict['storageLocation'], eid_val), 'w')
                            f.write(repr(outputList)); f.close()
                        except:
                            self.emit(QtCore.SIGNAL('pdf_file'),
                                      '<b>Indexing failed for this file</b><br><br>')
                            self.emit(QtCore.SIGNAL('no_pdf_file'), '<i>%s</i><br>' % pdf_filename)
                            continue
                    else:
                        self.emit(QtCore.SIGNAL('no_pdf_file'), '<i>%s</i><br>' % pdf_filename)
                        return
        self.emit(QtCore.SIGNAL('pdf_file'), '<br><br>%s<br>' % 'All pdfs done.')
        pdf_downloader.timer_stop(self.time_st)
        

    def get_doi(self, pdf_name):
        import pdf_downloader
        # In case there are spaces in the pdf name.
        pdf_name_no_space = self.remove_title_chars(pdf_name)
        #
        re_val_array = []
        re_val_array.append(re.compile('doi\s(.*?)\s', re.IGNORECASE))
        re_val_array.append(re.compile('doi:\s*(.*?)\s', re.IGNORECASE))
        re_val_array.append(re.compile('doi\s*(.*?)\s', re.IGNORECASE))
        re_val_array.append(re.compile('\s*(10\.[0-9]{3,15}.*?)\s', re.IGNORECASE))
        #pdf_name = re.escape(pdf_name)
        osFunc = "%s %s -" % (self.config_dict['pdfToTextLocation'], pdf_name_no_space)
        f = os.popen(osFunc)
        f = f.read()
        search_term = self.search_doi_re_array(f, re_val_array)
        # first pass with command line pdf extraction
        if len(search_term) > 0:
            if len(search_term[0]) > 100:
                try:
                    self.emit(QtCore.SIGNAL('pdf_file'), '%s<br>' %
                              '- No doi with first pass - xpdf')
                    temp_file_pre = self.tempdir + 'temp'
                    temp_filename = temp_file_pre + '.pdf'
                    shutil.copyfile(pdf_name, temp_filename)
                    f = os.system('%s %s' % (self.config_dict['pdfToTextLocation'], temp_filename))
                    while f: pass
                    x = open(temp_file_pre + '.txt').read()
                    search_term = self.search_doi_re_array(x, re_val_array)
                    search_term = self.clean_special_chars(search_term)
                    return search_term
                except:
                    pass
            else:
                search_term = self.clean_special_chars(search_term)
                return search_term
            
        # In case there was a problem with xpdf extraction at the command line
        try:
            
            self.emit(QtCore.SIGNAL('pdf_file'), '%s<br>' %
                      '- No doi with second pass - xpdf')
            temp_file_pre = self.tempdir + 'temp'
            temp_filename = temp_file_pre + '.pdf'
            shutil.copyfile(pdf_name, temp_filename)
            f = os.system('%s %s' % (self.config_dict['pdfToTextLocation'], temp_filename))
            while f: pass
            x = open(temp_file_pre + '.txt').read()
            search_term = self.search_doi_re_array(x, re_val_array)
            if len(search_term) > 0:
                search_term = self.clean_special_chars(search_term)
                return search_term
        except:
            pass

        # In case xpdf did not work.
        try:
            self.emit(QtCore.SIGNAL('pdf_file'), '%s<br>' %
                              '- No doi with third pass - xpdf')
            f = self.getPDFContent(pdf_name)
            if not f: return None
            search_term = self.search_doi_re_array(f, re_val_array)
            if len(search_term) > 0:
                search_term = self.clean_special_chars(search_term)
                return search_term
        except:
            return None

    ## {{{ http://code.activestate.com/recipes/511465/ (r2)
    def getPDFContent(self, pdf_name):
        content = ""
        # Load PDF into pyPDF
        pdf = pyPdf.PdfFileReader(file(pdf_name, "rb"))
        # Iterate pages
        for i in range(0, pdf.getNumPages()):
            # Extract text from page and add to content
            content += pdf.getPage(i).extractText() + "\n"
        # Collapse whitespace
        content = ' '.join(content)
        #content = " ".join(content.replace(u"\xa0", " ").strip().split())
        #content = content.encode("ascii", "xmlcharrefreplace")
        if not content: return None
        return content

    def clean_special_chars(self, search_term_array):
        doi_return = ''
        try:
            for i in search_term_array[0]:
                doi_return += unicode(i)
        except:
            pass
        return doi_return

    def remove_title_special_chars(self, search_term):
        return_val = []
        for j in search_term.split():
            try:
                return_val += [unicode(j)]
            except:
                continue
        return ' '.join(return_val)
    
    def get_one_search_item(self, search_term):
        print search_term
        database = 'pubmed'
        Entrez.email = 'a.varier@gmail.com'
        handle = Entrez.esearch(db=database, term=search_term,
                                tool="BioPython, Bio.Entrez", retmax = 1)
        records = Entrez.read(handle)
        idlist = records['IdList']
        handle = Entrez.efetch(db="pubmed", id=idlist, rettype="medline", retmode="text")
        records = Medline.parse(handle)
        records = list(records)
        details = []
        details_titles = ['PMRecord', 'Title',
                          'Authors', 'Journal', 'Issue', 'Pages', 'Year']
        for record in records:
            tempRecord = {}
            tempRecord['Title'] = record.get("TI")
            tempRecord['PMRecord'] = record.get("PMID")
            tempRecord['Authors'] = ", ".join(record.get('AU'))
            if record.get("JT"):
                tempRecord['Journal'] = record.get("JT")
            elif record.get("CTI"):
                tempRecord['Journal'] = ''.join(record["CTI"])
            else:
                tempRecord['Journal'] = ''
            issue = record.get('VI') if record.get('VI') else ''
            issue = '%s (%s)' % (issue, record.get('IP')) if record.get('IP') else issue

            pages = record.get('PG') if record.get('PG') else ''
            pages = record.get('IS') if pages == '' else pages
            tempRecord['Pages'] = pages
            tempRecord['Issue'] = issue
            tempRecord['Year'] = record.get('DA')[:4]
            tempRecord['ArticleRecord'] = ''
            tempRecord['ArticleLink'] = ('http://www.hubmed.org/fulltext.cgi?uids=' +
                                         tempRecord['PMRecord'])
            tempRecord['EID'] = tempRecord['PMRecord']
            tempRecord['PMC'] = record.get('PMC') if record.get('PMC') else ''
            details += [tempRecord]
        return details

    def search_doi_re_array(self, stringVal, reArray):
        for re_val in reArray:
            search_term = re_val.findall(stringVal)
            if len(search_term) > 0:
                if not search_term[0].strip() == 'ng':
                    return search_term
        return []
    
    def try_to_get_title(self, pdf_name):
        '''
        If there was no doi number at least try to get a title from the article. Ill assume
        (falsely) that the title has at least 7 words. If it doesnt, then get the first line that has 7 words. This could be in the abstract. It doesnt matter.
        '''
        pdf_name_no_space = self.remove_title_chars(pdf_name)
        osFunc = "%s %s -" % (self.config_dict['pdfToTextLocation'], pdf_name_no_space)
        author_re = re.compile('\s.*[A-Z]\.')
        f = os.popen(osFunc)
        f = f.read()
        x = [i.split('.')[0] for i in f.split('\n')
             if ((len(i.split()) > 5)
                 and not (len(author_re.findall(i)) > 0)
                 and not ('Vol.' in i)
                 and not ('*' in i)
                 and not ('Departm' in i)
                 and not ('Institute' in i)
                 and not ('Universit' in i)
                 and not ('Laborat' in i)
                 and not ('Europe' in i)
                 and not ('VOL.' in i)
                 and not ('Journal' in i)
                 and not ('JOURNAL' in i)
                 and not ('LETTERS' in i)
                 and not ('.pdf' in i))]
        title_val = x[0]
        title_val = title_val.replace('Abstract:', '')
        title_val = title_val.replace("ABSTRACT:", '')
        title_val = self.remove_title_special_chars(title_val)
        return title_val


    def index_previously_indexed(self, root, base_filename, pdf_filename):
        try:
            f = open(root + '/' + base_filename + '.dxt').read()
        except:
            return None
        record = eval(f)
        self.emit(QtCore.SIGNAL('pdf_file'),
              '- <i>%s</i><br>' % 'Previous CitationControl record found.')
        title = record['Title']
        new_record = self.get_one_search_item(title)
        record_id = new_record['EID']
        record['PMRecord'] = new_record['PMRecord']
        record['EID'] = record_id
        record['PMC'] = new_record['PMC']
        f = open('%s%s.dxt' % (self.config_dict['storageLocation'], record_id), 'w')
        f.write(repr(record)); f.close()

        if self.config_dict['savePdfsCheck'] == 'Yes':
           self.emit(QtCore.SIGNAL('pdf_file'),
                  '- <i>%s</i><br>' % 'Copying pdf to storage.') 
           new_pdf_name = '%s%s.pdf' % (
               self.config_dict['storageLocation'], record_id)
           shutil.copyfile(pdf_filename, new_pdf_name)

        self.emit(QtCore.SIGNAL('pdf_file'),
                      '- <i>%s</i><br>' % 'Copying searchable text.')
        try:
            old_text_filename = root + '/' + base_filename + '.txt'
            new_text_filename = '%s%s.txt' % (
               self.config_dict['storageLocation'], record_id)
            shutil.copyfile(old_text_filename, new_text_filename)
        except:
            try:
                self.emit(QtCore.SIGNAL('pdf_file'),
                          '- <i>%s</i><br>' %
                          'Previous text not found. Trying extraction again.')
                self.convert_pdf_to_text(pdf_filename, record_id)
            except:
                self.emit(QtCore.SIGNAL('pdf_file'),
                          '- <i>%s</i><br>' %
                          'Text extraction not possible.')
                pass

        self.emit(QtCore.SIGNAL('pdf_file'),
                      '- <i>%s</i><br>' % 'Copying previous notes.')
        try:
            old_nxt_filename = root + '/' + base_filename + '.nxt'
            new_nxt_filename = '%s%s.nxt' % (
               self.config_dict['storageLocation'], record_id)
            shutil.copyfile(old_nxt_filename, new_nxt_filename)
        except:
            pass
        self.emit(QtCore.SIGNAL('pdf_file'),
                      '<br>')
        return True

    def remove_title_chars(self, name):
        name_no_space = name.replace(' ', '\ ')
        name_no_space = name_no_space.replace('(', '\(')
        name_no_space = name_no_space.replace(')', '\)')
        return name_no_space
    
    
def remove_title_special_chars(search_term):
    return_val = []
    for j in search_term.split():
        try:
            return_val += [unicode(j)]
        except:
            continue
    return ' '.join(return_val)
    
def getPDFContent(pdf_name):
    content = ""
    # Load PDF into pyPDF
    pdf = pyPdf.PdfFileReader(file(pdf_name, "rb"))
    # Iterate pages
    for i in range(0, pdf.getNumPages()):
        # Extract text from page and add to content
        content += pdf.getPage(i).extractText() + "\n"
    # Collapse whitespace
    content = " ".join(content.replace(u"\xa0", " ").strip().split())
    content = content.encode("ascii", "xmlcharrefreplace")
    if not content: return None
    return content
